package ac.manchester.cs.afzal.autoservd.Descriptors;

import ac.manchester.cs.afzal.autoservd.file_handler.File_Manager;
import ac.manchester.cs.afzal.autoservd.regular_expressions.String_Manipulator;
import ac.manchester.cs.afzal.autoservd.vector_handler.Vector_Builder;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Iterator;
import java.util.Properties;
import java.util.StringTokenizer;
import java.util.Vector;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 *
 * @author hamafz
 */
public class Descriptor_Processor
{

    private String PENN_FILES_LOCATION_FETA = "PENN_FILES_LOCATION_FETA";
    private String PENN_FILES_LOCATION_PROVIDERS = "PENN_FILES_LOCATION_PROVIDERS";
    private String PENN_FILES_LOCATION_CORPUS = "PENN_FILES_LOCATION_CORPUS";
    private String STOPWORDS_LOCATION = "STOPWORDS_LOCATION";
    private String STARTWORDS_LOCATION = "STARTWORDS_LOCATION";
    private String CATEGORY = "CATEGORY";
    private String DESCRIPTORS_OUTPUT_XML = "DESCRIPTORS_OUTPUT_XML";


    private Vector<String> stopwords = new Vector();
    private Vector<String> startwords = new Vector();

    private SQL_Handler_Descriptors sqlHandler2;
    private SQL_Handler_Descriptors sqlHandler1;

    private File_Manager fileManager;
    private Vector_Builder vectorBuilder;
    private String_Manipulator strMan;
    private Properties props;

    String[] doc_table1 =
    {
        "Descriptor", "Filtered_Descriptor", "Freq_1", "Freq_2", "Category"
    };
    String[] doc_table2 =
    {
        "Descriptor", "Filtered_Descriptor", "D_Sen_ID"
    };
    String[] doc_table_temp =
    {
        "Value", "Filtered_Value", "Category"
    };

    final String xmlDeclaration = "<?xml version=\"1.0\" encoding=\"iso-8859-1\"?>";
    final String xmlSchema = "<Descriptors xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:noNamespaceSchemaLocation=\"corpus.xsd\">";


    /**
     *
     */
    public Descriptor_Processor()
    {
        fileManager = new File_Manager();
        vectorBuilder = new Vector_Builder();
        strMan = new String_Manipulator();

        read_stopwords(STOPWORDS_LOCATION);
        read_startwords(STOPWORDS_LOCATION);

        this.init_database(2);

        props = new Properties();
        try
        {
            props.load(new FileInputStream("AutoServD.properties"));
        }
        catch (IOException ex)
        {
            Logger.getLogger(Descriptor_Processor.class.getName()).log(Level.SEVERE, null, ex);
        }

        PENN_FILES_LOCATION_FETA = props.getProperty(PENN_FILES_LOCATION_FETA);
        PENN_FILES_LOCATION_PROVIDERS = props.getProperty(PENN_FILES_LOCATION_PROVIDERS);
        PENN_FILES_LOCATION_CORPUS = props.getProperty(PENN_FILES_LOCATION_CORPUS);
        STOPWORDS_LOCATION = props.getProperty(STOPWORDS_LOCATION);
        STARTWORDS_LOCATION = props.getProperty(STARTWORDS_LOCATION);
        CATEGORY = props.getProperty(CATEGORY);
        DESCRIPTORS_OUTPUT_XML = props.getProperty(DESCRIPTORS_OUTPUT_XML);

    }

    /**
     * This method retrieves the noun phrases parsed files of Feta Descriptions, Descriptions from Online Service Providers
     * and Corpus and load into ___temp table, and __sentence_descriptors tables respectively
     */
    public void extract_nps_from_text_files()
    {
        File file_dir;

        String main_directory = "";

        String[] list_dirs = new String[]
        {
        };

        if (CATEGORY.equals("Feta Descriptions"))
        {
            sqlHandler2.reset_tables(new String[]
                    {
                        "___bioinformatics_service_descriptors", "___temp"
                    });

            file_dir = new File(PENN_FILES_LOCATION_FETA);
            list_dirs = file_dir.list();
            main_directory = PENN_FILES_LOCATION_FETA;
        }
        else if (CATEGORY.equals("Service Providers"))
        {
            sqlHandler2.reset_tables(new String[]
                    {
                        "___bioinformatics_service_descriptors", "___temp"
                    });

            file_dir = new File(PENN_FILES_LOCATION_PROVIDERS);
            list_dirs = file_dir.list();
            main_directory = PENN_FILES_LOCATION_PROVIDERS;
        }
        else if (CATEGORY.equals("Corpus"))
        {
            sqlHandler2.reset_tables(new String[]
                    {
                        "__sentence_descriptors"
                    });
            file_dir = new File(PENN_FILES_LOCATION_CORPUS);
            list_dirs = file_dir.list();
            main_directory = PENN_FILES_LOCATION_CORPUS;

        }
        else
        {
            System.out.println("Wrong entry of Category: The valid options are 'Service Providers', 'Feta Descriptions' and 'Corpus'");
            System.exit(1);
        }


        for (String directory_name : list_dirs)
        {
            String file_contents = "";
            File file_doc = new File(main_directory + directory_name);
            String[] list_files = file_doc.list();


            for (String file_name : list_files)
            {
                String distinct_sentence_id = fileManager.strip_file_name_from_file_name_plus_extension(file_name, "xml");
                File file_per_term = new File(main_directory + directory_name + "//" + file_name);
                file_per_term = file_per_term.getAbsoluteFile();

                StringBuffer file_contents_buffer = fileManager.fileReader(file_per_term);
                file_contents = file_contents_buffer.toString();

                int startTab = 0;
                int endTab = 0;
                int tabCheckerIndex = 0;
                String tempTabValue = "";
                String fieldValue = "";

                while ((startTab = file_contents.indexOf("<st_", startTab)) != -1)
                {
                    endTab = file_contents.indexOf(">", startTab);
                    tempTabValue = file_contents.substring(startTab + 4, endTab);

                    if (tempTabValue.equals("NP"))
                    {
                        if ((tabCheckerIndex = file_contents.indexOf("</st_NP", endTab + 1)) != -1)
                        {
                            fieldValue = file_contents.substring(endTab + 1, tabCheckerIndex);

                            if (fieldValue.contains("<st_NP"))
                            {
                                startTab = startTab + 6;
                            }
                            else
                            {
                                String descriptor_np = strMan.pattern_replacer(fieldValue, "<st_(.){0,5}>", " ");
                                descriptor_np = strMan.pattern_replacer(descriptor_np, "</st_(.){0,5}>", " ");
                                descriptor_np = descriptor_np.replace("  ", " ");
                                descriptor_np = descriptor_np.replace("   ", " ");

                                if (is_nlp_fit(descriptor_np))
                                {
                                    String filtered_descriptor_np = filter_stop_words_from_one_element(descriptor_np);

                                    if (CATEGORY.equals("Corpus"))
                                    {
                                        String[] doc_table_value_temp =
                                        {
                                            descriptor_np, filtered_descriptor_np, distinct_sentence_id
                                        };
                                        sqlHandler2.insert_table("__sentence_descriptors", doc_table2, doc_table_value_temp);

                                    }
                                    else
                                    {
                                        String[] doc_table_value_temp =
                                        {
                                            descriptor_np, filtered_descriptor_np, CATEGORY
                                        };
                                        sqlHandler2.insert_table("__temp", doc_table_temp, doc_table_value_temp);
                                    }
                                }
                                startTab = startTab + 7;
                            }
                        }
                        else
                        {
                            startTab = startTab + 6;
                        }
                    }
                    else
                    {
                        startTab = startTab + 6;
                    }
                }
                System.out.println("FileName : " + file_per_term);
            }
        }
    }

    /**
     * This method loads the filtered service descriptors from a text file into the __bioinofrmatics-service_descriptors table
     */
    public void load_filtered_descriptors_from_csv_file()
    {
        sqlHandler2.reset_tables(new String[]
                {
                    "___bioinformatics_service_descriptors"
                });

        Iterator startwords_iter = startwords.iterator();
        while (startwords_iter.hasNext())
        {
            String descriptor = startwords_iter.next().toString();
            String freq_2 = sqlHandler2.get_descriptor_freq_in_corpus_sentence_descriptors(descriptor);


            String[] doc_table_value_temp =
            {
                descriptor, descriptor, "0", freq_2, "manually_filtered"
            };
            sqlHandler2.insert_table("___bioinformatics_service_descriptors", doc_table1, doc_table_value_temp);
        }
    }

    /**
     * This method finds the most relavant 300 terms from Results_temp table of BMC_2009 database and loads them into ___temp table.
     */
    public void extract_top_300_terms()
    {
        Vector<String> top_300 = new Vector();

        top_300 = sqlHandler1.get_top_300_from_results_final();
        for (String term: top_300)
        {
            String filtered_term = filter_stop_words_from_one_element(term);
            String[] doc_table_value_temp =
            {
                term, filtered_term, "top_300"
            };
            sqlHandler2.insert_table("___temp", doc_table_temp, doc_table_value_temp);
        }
    }

    /**
     * This methods retrieves the service descriptors from ___temp table and loads them into ___bioinformatics_service_descriptors table after
     * finding their respective frequencies in Corpus
     */
    public void export_descriptors_to_bioinformatics_service_descriptors_table()
    {
        sqlHandler2.get_fields_from_temp_table();
        Vector<String> descriptors = sqlHandler2.get_descriptors();
        Vector<String> descriptors_count = sqlHandler2.get_descriptors_freq_1();
        Vector<String> filtered_descriptors = sqlHandler2.get_descriptors_filtered();
        Vector<String> descriptors_category = sqlHandler2.get_descriptors_category();

        for (int i = 0; i < descriptors.size(); i++)
        {
            String descriptor = descriptors.elementAt(i);
            String filtered_descriptor = filtered_descriptors.elementAt(i);
            String descriptor_category = descriptors_category.elementAt(i);
            String freq_1 = descriptors_count.elementAt(i);
            String freq_2 = sqlHandler2.get_descriptor_freq_in_corpus_sentence_descriptors(filtered_descriptor);

            String[] doc_table_value1 =
            {
                descriptor, filtered_descriptor, freq_1, freq_2, descriptor_category
            };
            sqlHandler2.insert_table("___bioinformatics_service_descriptors", doc_table1, doc_table_value1);
        }
    }

    /**
     * Exports the Bioinformatics Service Descriptors from Table to XML format (stored in Resources directory
     */
    public void export_to_xml()
    {
        sqlHandler2.get_all_descriptor_fields();

        Vector descriptors = sqlHandler2.get_descriptors();
        Vector descriptors_filtered = sqlHandler2.get_descriptors_filtered();
        Vector descriptors_freq_1 = sqlHandler2.get_descriptors_freq_1();
        Vector descriptors_freq_2 = sqlHandler2.get_descriptors_freq_2();
        Vector descriptors_category = sqlHandler2.get_descriptors_category();

        StringBuffer strBufWritor = new StringBuffer();
        strBufWritor.append(xmlDeclaration);
        strBufWritor.append(System.getProperty("line.separator"));
        strBufWritor.append(xmlSchema);
        strBufWritor.append(System.getProperty("line.separator"));


        for (int i = 0; i < descriptors.size(); i++)
        {
            strBufWritor.append("<Descriptor id = '" + i + "'>");
            strBufWritor.append(System.getProperty("line.separator"));

            String descriptor = descriptors.elementAt(i).toString();
            String descriptor_filtered = descriptors_filtered.elementAt(i).toString();
            String descriptor_freq_1 = descriptors_freq_1.elementAt(i).toString();
            String descriptor_freq_2 = descriptors_freq_2.elementAt(i).toString();
            String descriptor_category = descriptors_category.elementAt(i).toString();

            strBufWritor.append("<Value>");
            strBufWritor.append(descriptor);
            strBufWritor.append("</Value>");
            strBufWritor.append(System.getProperty("line.separator"));

            strBufWritor.append("<Filtered_Value>");
            strBufWritor.append(descriptor_filtered);
            strBufWritor.append("</Filtered_Value>");
            strBufWritor.append(System.getProperty("line.separator"));

            strBufWritor.append("<Freq_1>");
            strBufWritor.append(descriptor_freq_1);
            strBufWritor.append("</Freq_1>");
            strBufWritor.append(System.getProperty("line.separator"));

            strBufWritor.append("<Freq_2>");
            strBufWritor.append(descriptor_freq_2);
            strBufWritor.append("</Freq_2>");
            strBufWritor.append(System.getProperty("line.separator"));

            strBufWritor.append("<Descriptor_category>");
            strBufWritor.append(descriptor_category);
            strBufWritor.append("</Descriptor_category>");
            strBufWritor.append(System.getProperty("line.separator"));

            strBufWritor.append("</Descriptor>");
            strBufWritor.append(System.getProperty("line.separator"));
        }

        strBufWritor.append("</Descriptors>");
        strBufWritor.append(System.getProperty("line.separator"));
        String file_contents = strBufWritor.toString();
        fileManager.fileWriter(file_contents, DESCRIPTORS_OUTPUT_XML);
    }

    /**
     * Reads stopwords list from Resources\stopwords.txt
     * @param path
     * @return
     */
    public void read_stopwords(String path)
    {
        String stopwords_str = fileManager.fileReader(path).toString();
        StringTokenizer st1 = new StringTokenizer(stopwords_str, "\r\n");
        while (st1.hasMoreElements())
        {
            String term = st1.nextToken();
            term = term.trim();
            term = term.toLowerCase();
            stopwords.add(term);
        }
    }

    /**
     * Reads filtered descriptos from  list from Resources\startlist.txt
     * "The naming convention is a bit wierd .. in contrast to Stopwords, it is named as Startlist 
     * @param path
     * @return
     */
    public void read_startwords(String path)
    {
        String startwords_str = fileManager.fileReader(path).toString();
        StringTokenizer st1 = new StringTokenizer(startwords_str, "\r\n");
        while (st1.hasMoreElements())
        {
            String term = st1.nextToken();
            term = term.trim();
            term = term.toLowerCase();
            startwords.add(term);
        }
    }

    /**
     * Filters the stopwords from a single string
     * @param element
     * @return
     */
    public String filter_stop_words_from_one_element(String element)
    {
        Vector<String> temp_np = new Vector();
        temp_np = vectorBuilder.string_to_vector(element, " ");
        temp_np.removeAll(stopwords);
        String processed_element = vectorBuilder.vector_to_string(temp_np, " ");
        processed_element.trim();
        processed_element = (processed_element.length() < 4) ? "EMPTY" : processed_element;

        return processed_element;
    }

    protected void init_database(int count)
    {
        sqlHandler1 = new SQL_Handler_Descriptors();
        sqlHandler1.initialize_parameters("jdbc:mysql://localhost:3306/BMC_2009", "root", "admin");
        sqlHandler1.connect_db();

        if (count == 2)
        {
            sqlHandler2 = new SQL_Handler_Descriptors();
            sqlHandler2.initialize_parameters("jdbc:mysql://localhost:3306/BMC_2009_2", "root", "admin");
            sqlHandler2.connect_db();
        }

    }

    public boolean is_nlp_fit(String np)
    {
        if ((np.length() > 50) || (np.contains("\'")) ||
                (np.length() < 4) || (np.contains("\"")) ||
                (np.contains("http")) || (np.contains("``")) ||
                (np.contains("&lt")) || (np.matches("^[0-9]")) ||
                (np.contains("</")) || (np.contains("@")))
        {
            return false;
        }
        else
        {
            return true;
        }
    }
}
